﻿// --------------------------------------------------------------------------------------------------------------------
// <copyright file="TextCleaner.cs" company="Arizona State University">
//   Copyright (C) 2011-2012 Sukru Tikves and Arizona State University
//   
//   Released under Microsoft Public License (Ms-PL)
//   See http://cipsdata.codeplex.com/license for up to date license 
//   informaiton.
// </copyright>
// <summary>
//   The text cleaner.
// </summary>
// --------------------------------------------------------------------------------------------------------------------

namespace Cips.DataMining
{
    using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.Diagnostics.CodeAnalysis;
    using System.Globalization;
    using System.Linq;
    using System.Text;
    using System.Text.RegularExpressions;

    /// <summary>
    /// The text cleaner.
    /// </summary>
    public static class TextCleaner
    {
        /// <summary>
        /// The separators.
        /// </summary>
        static readonly char[] Separators = { ' ', '\t', '\n', '\r', '`', '~', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '-', '=', '_', '+', '\'', '"', ';', ':', ',', '.', '/', '\\', '?', '[', ']' };

        /// <summary>
        /// The parser state.
        /// </summary>
        [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1602:EnumerationItemsMustBeDocumented", Justification = "Reviewed. Suppression is OK here.")]
        enum ParserState
        {
            Initial,
            Keyword,
            Number,
            Other,
        }

        /// <summary>
        /// Gets the words from a document.
        /// </summary>
        /// <param name="document">The document.</param>
        /// <returns>A list of words.</returns>
        public static IEnumerable<string> GetWords(string document)
        {
            return document.Split(Separators, StringSplitOptions.RemoveEmptyEntries);
        }

        /// <summary>
        /// Converts a list of word into lower case.
        /// </summary>
        /// <param name="document">The input document.</param>
        /// <returns>The converted strings.</returns>
        public static IEnumerable<string> AllLowerCase(this IEnumerable<string> document)
        {
            return from word in document select word.ToLower(CultureInfo.InvariantCulture);
        }

        /// <summary>
        /// Tokenizes the specified input.
        /// </summary>
        /// <param name="input">The input.</param>
        /// <returns>A token stream representing the input.</returns>
        public static IEnumerable<string> Tokenize(string input)
        {
            var keyword = new StringBuilder();
            var state = ParserState.Initial;
            var email = new Regex(@"([\w-\.]+)@((?:[\w]+\.)+)([a-zA-Z]{2,4})");
            var url = new Regex(@"(ht|f)tps?:\/\/w{0,3}[a-zA-Z0-9_\-.:#/~&}?%\(\),\{\}]+");

            for (var index = 0; index < input.Length; index++)
            {
                var ch = input[index];
                var next = index < (input.Length - 1) ? input[index + 1] : (char)0;

                switch (state)
                {
                    case ParserState.Initial:
                        switch (char.GetUnicodeCategory(ch))
                        {
                            case UnicodeCategory.UppercaseLetter:
                            case UnicodeCategory.LowercaseLetter:
                                keyword.Append(ch);
                                state = ParserState.Keyword;
                                break;

                            case UnicodeCategory.SpaceSeparator:
                            case UnicodeCategory.DashPunctuation:
                                break;

                            case UnicodeCategory.DecimalDigitNumber:
                                state = ParserState.Number;
                                break;

                            case UnicodeCategory.Control:
                            case UnicodeCategory.OtherPunctuation:
                            case UnicodeCategory.OpenPunctuation:
                            case UnicodeCategory.ClosePunctuation:
                            case UnicodeCategory.InitialQuotePunctuation:
                            case UnicodeCategory.FinalQuotePunctuation:
                            case UnicodeCategory.MathSymbol:
                            case UnicodeCategory.ConnectorPunctuation:
                            case UnicodeCategory.ModifierLetter:
                            case UnicodeCategory.ModifierSymbol:
                            case UnicodeCategory.OtherNumber:
                            case UnicodeCategory.CurrencySymbol:
                                break;

                            case UnicodeCategory.OtherLetter:
                                state = ParserState.Other;
                                break;

                            default:
                                break;
                        }

                        break;

                    case ParserState.Keyword:
                        switch (char.GetUnicodeCategory(ch))
                        {
                            case UnicodeCategory.UppercaseLetter:
                            case UnicodeCategory.LowercaseLetter:
                            case UnicodeCategory.ModifierLetter:
                            case UnicodeCategory.ModifierSymbol:
                            case UnicodeCategory.CurrencySymbol:
                                keyword.Append(ch);
                                break;

                            case UnicodeCategory.SpaceSeparator:
                            case UnicodeCategory.DashPunctuation:
                            case UnicodeCategory.Control:
                            case UnicodeCategory.OpenPunctuation:
                            case UnicodeCategory.ClosePunctuation:
                            case UnicodeCategory.OtherNumber:
                                state = ParserState.Initial;

                                if (email.IsMatch(keyword.ToString()))
                                    yield return "<email>";
                                else if(keyword.Length != 1)
                                    yield return keyword.ToString();

                                keyword.Clear();
                                break;

                            case UnicodeCategory.DecimalDigitNumber:
                                state = ParserState.Number;
                                index--;
                                break;

                            case UnicodeCategory.OtherLetter:
                                state = ParserState.Other;
                                index--;
                                break;

                            case UnicodeCategory.OtherPunctuation:
                            case UnicodeCategory.FinalQuotePunctuation:
                            case UnicodeCategory.InitialQuotePunctuation:
                            case UnicodeCategory.MathSymbol:
                            case UnicodeCategory.ConnectorPunctuation:
                                if (ch == '\'' || ch == '’' || ch == '‘' || ch == '‟')
                                {
                                    if (char.IsLetter(next))
                                        keyword.Append('\'');
                                    else
                                        goto case UnicodeCategory.SpaceSeparator;

                                    break;
                                }

                                if (ch == '@' || ch == '.' || ch == '&' || ch == '_' || ch == '*' || ch == '«' || ch == '»' || ch == '·' || ch == '†' || ch == '‰' || ch == '#' || ch == '%'  || ch == '+' || ch == '~')
                                {
                                    if (char.IsLetter(next))
                                        keyword.Append(ch);
                                    else
                                        goto case UnicodeCategory.SpaceSeparator;

                                    break;
                                }

                                if (ch == ':' && (keyword.ToString() == "http" || keyword.ToString() == "https" || keyword.ToString() == "ftp"))
                                {
                                    var match = url.Match(input, index - keyword.Length);

                                    if (match.Success)
                                    {
                                        index += match.Length - keyword.Length;

                                        keyword.Clear();
                                        state = ParserState.Initial;

                                        yield return "<url>";

                                        break;
                                    }
                                }

                                if (ch == ',' || ch == ':' || ch == ';' || ch == '?' || ch == '!' || ch == '"' || ch == '/' || ch == '”' || ch == '¡' || ch == '“' || ch == '…' || ch == '|' || ch == '\\' || ch == '=' || ch == '>' || ch == '<')
                                    goto case UnicodeCategory.SpaceSeparator;

                                if (next == 0 || char.IsWhiteSpace(next) || char.IsPunctuation(next) || char.IsSymbol(next) || char.IsNumber(next))
                                    goto case UnicodeCategory.SpaceSeparator;

                                goto default;

                            default:
                                break;
                        }

                        break;

                    case ParserState.Number:
                        switch (char.GetUnicodeCategory(ch))
                        {
                            case UnicodeCategory.SpaceSeparator:
                            case UnicodeCategory.DashPunctuation:
                            case UnicodeCategory.UppercaseLetter:
                            case UnicodeCategory.LowercaseLetter:
                            case UnicodeCategory.OtherPunctuation:
                            case UnicodeCategory.ClosePunctuation:
                            case UnicodeCategory.Control:
                            case UnicodeCategory.FinalQuotePunctuation:
                            case UnicodeCategory.OpenPunctuation:
                            case UnicodeCategory.OtherLetter:
                            case UnicodeCategory.ConnectorPunctuation:
                            case UnicodeCategory.MathSymbol:
                            case UnicodeCategory.ModifierSymbol:
                                state = ParserState.Initial;
                                index--;

                                yield return "<number>";
                                break;

                            case UnicodeCategory.DecimalDigitNumber:
                            case UnicodeCategory.OtherNumber:
                            case UnicodeCategory.CurrencySymbol:
                                break;

                            default:
                                state = ParserState.Initial;
                                index--;

                                yield return "<number>";
                                break;
                        }

                        break;

                    case ParserState.Other:
                        if (char.GetUnicodeCategory(ch) != UnicodeCategory.OtherLetter)
                        {
                            state = ParserState.Initial;
                            index--;

                            yield return "<other>";
                        }

                        break;

                    default:
                        throw new ArgumentOutOfRangeException();
                }
            }

            switch (state)
            {
                case ParserState.Initial:
                    break;

                case ParserState.Keyword:
                    if (email.IsMatch(keyword.ToString()))
                        yield return "<email>";
                    else if (keyword.Length != 1)
                        yield return keyword.ToString();
                    break;

                case ParserState.Number:
                    yield return "<number>";
                    break;

                case ParserState.Other:
                    yield return "<other>";
                    break;

                default:
                    throw new ArgumentOutOfRangeException();
            }
        }
    }
}
